<!-- HTML header for doxygen 1.8.6-->
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<meta http-equiv="Content-Type" content="text/xhtml;charset=UTF-8"/>
<meta http-equiv="X-UA-Compatible" content="IE=9"/>
<meta name="generator" content="Doxygen 1.8.13"/>
<title>OpenCV: How to run deep networks in browser</title>
<link href="../../opencv.ico" rel="shortcut icon" type="image/x-icon" />
<link href="../../tabs.css" rel="stylesheet" type="text/css"/>
<script type="text/javascript" src="../../jquery.js"></script>
<script type="text/javascript" src="../../dynsections.js"></script>
<script type="text/javascript" src="../../tutorial-utils.js"></script>
<link href="../../search/search.css" rel="stylesheet" type="text/css"/>
<script type="text/javascript" src="../../search/searchdata.js"></script>
<script type="text/javascript" src="../../search/search.js"></script>
<script type="text/x-mathjax-config">
  MathJax.Hub.Config({
    extensions: ["tex2jax.js", "TeX/AMSmath.js", "TeX/AMSsymbols.js"],
    jax: ["input/TeX","output/HTML-CSS"],
});
//<![CDATA[
MathJax.Hub.Config(
{
  TeX: {
      Macros: {
          matTT: [ "\\[ \\left|\\begin{array}{ccc} #1 & #2 & #3\\\\ #4 & #5 & #6\\\\ #7 & #8 & #9 \\end{array}\\right| \\]", 9],
          fork: ["\\left\\{ \\begin{array}{l l} #1 & \\mbox{#2}\\\\ #3 & \\mbox{#4}\\\\ \\end{array} \\right.", 4],
          forkthree: ["\\left\\{ \\begin{array}{l l} #1 & \\mbox{#2}\\\\ #3 & \\mbox{#4}\\\\ #5 & \\mbox{#6}\\\\ \\end{array} \\right.", 6],
          forkfour: ["\\left\\{ \\begin{array}{l l} #1 & \\mbox{#2}\\\\ #3 & \\mbox{#4}\\\\ #5 & \\mbox{#6}\\\\ #7 & \\mbox{#8}\\\\ \\end{array} \\right.", 8],
          vecthree: ["\\begin{bmatrix} #1\\\\ #2\\\\ #3 \\end{bmatrix}", 3],
          vecthreethree: ["\\begin{bmatrix} #1 & #2 & #3\\\\ #4 & #5 & #6\\\\ #7 & #8 & #9 \\end{bmatrix}", 9],
          cameramatrix: ["#1 = \\begin{bmatrix} f_x & 0 & c_x\\\\ 0 & f_y & c_y\\\\ 0 & 0 & 1 \\end{bmatrix}", 1],
          distcoeffs: ["(k_1, k_2, p_1, p_2[, k_3[, k_4, k_5, k_6 [, s_1, s_2, s_3, s_4[, \\tau_x, \\tau_y]]]]) \\text{ of 4, 5, 8, 12 or 14 elements}"],
          distcoeffsfisheye: ["(k_1, k_2, k_3, k_4)"],
          hdotsfor: ["\\dots", 1],
          mathbbm: ["\\mathbb{#1}", 1],
          bordermatrix: ["\\matrix{#1}", 1]
      }
  }
}
);
//]]>
</script><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.0/MathJax.js"></script>
<link href="../../doxygen.css" rel="stylesheet" type="text/css" />
<link href="../../stylesheet.css" rel="stylesheet" type="text/css"/>
</head>
<body>
<div id="top"><!-- do not remove this div, it is closed by doxygen! -->
<div id="titlearea">
<!--#include virtual="/google-search.html"-->
<table cellspacing="0" cellpadding="0">
 <tbody>
 <tr style="height: 56px;">
  <td id="projectlogo"><img alt="Logo" src="../../opencv-logo-small.png"/></td>
  <td style="padding-left: 0.5em;">
   <div id="projectname">OpenCV
   &#160;<span id="projectnumber">4.5.2</span>
   </div>
   <div id="projectbrief">Open Source Computer Vision</div>
  </td>
 </tr>
 </tbody>
</table>
</div>
<!-- end header part -->
<!-- Generated by Doxygen 1.8.13 -->
<script type="text/javascript">
var searchBox = new SearchBox("searchBox", "../../search",false,'Search');
</script>
<script type="text/javascript" src="../../menudata.js"></script>
<script type="text/javascript" src="../../menu.js"></script>
<script type="text/javascript">
$(function() {
  initMenu('../../',true,false,'search.php','Search');
  $(document).ready(function() { init_search(); });
});
</script>
<div id="main-nav"></div>
<!-- window showing the filter options -->
<div id="MSearchSelectWindow"
     onmouseover="return searchBox.OnSearchSelectShow()"
     onmouseout="return searchBox.OnSearchSelectHide()"
     onkeydown="return searchBox.OnSearchSelectKey(event)">
</div>

<!-- iframe showing the search results (closed by default) -->
<div id="MSearchResultsWindow">
<iframe src="javascript:void(0)" frameborder="0" 
        name="MSearchResults" id="MSearchResults">
</iframe>
</div>

<div id="nav-path" class="navpath">
  <ul>
<li class="navelem"><a class="el" href="../../d9/df8/tutorial_root.html">OpenCV Tutorials</a></li><li class="navelem"><a class="el" href="../../d2/d58/tutorial_table_of_content_dnn.html">Deep Neural Networks (dnn module)</a></li>  </ul>
</div>
</div><!-- top -->
<div class="header">
  <div class="headertitle">
<div class="title">How to run deep networks in browser </div>  </div>
</div><!--header-->
<div class="contents">
<div class="textblock"><p><b>Prev Tutorial:</b> <a class="el" href="../../da/d9d/tutorial_dnn_yolo.html">YOLO DNNs</a></p>
<p><b>Next Tutorial:</b> <a class="el" href="../../dc/db1/tutorial_dnn_custom_layers.html">Custom deep learning layers support</a></p>
<table class="doxtable">
<tr>
<th align="right"></th><th align="left"></th></tr>
<tr>
<td align="right">Original author </td><td align="left">Dmitry Kurtaev </td></tr>
<tr>
<td align="right">Compatibility </td><td align="left">OpenCV &gt;= 3.3.1 </td></tr>
</table>
<h2>Introduction</h2>
<p>This tutorial will show us how to run deep learning models using OpenCV.js right in a browser. Tutorial refers a sample of face detection and face recognition models pipeline.</p>
<h2>Face detection</h2>
<p>Face detection network gets BGR image as input and produces set of bounding boxes that might contain faces. All that we need is just select the boxes with a strong confidence.</p>
<h2>Face recognition</h2>
<p>Network is called OpenFace (project <a href="https://github.com/cmusatyalab/openface">https://github.com/cmusatyalab/openface</a>). Face recognition model receives RGB face image of size <code>96x96</code>. Then it returns <code>128</code>-dimensional unit vector that represents input face as a point on the unit multidimensional sphere. So difference between two faces is an angle between two output vectors.</p>
<h2>Sample</h2>
<p>All the sample is an HTML page that has JavaScript code to use OpenCV.js functionality. You may see an insertion of this page below. Press <code>Start</code> button to begin a demo. Press <code>Add a person</code> to name a person that is recognized as an unknown one. Next we'll discuss main parts of the code.</p>
<!DOCTYPE html>

<html>

<head>
  <script async src="../../opencv.js" type="text/javascript"></script>
  <script src="../../utils.js" type="text/javascript"></script>

<script type='text/javascript'>
var netDet = undefined, netRecogn = undefined;
var persons = {};

//! [Run face detection model]
function detectFaces(img) {
  var blob = cv.blobFromImage(img, 1, {width: 192, height: 144}, [104, 117, 123, 0], false, false);
  netDet.setInput(blob);
  var out = netDet.forward();

  var faces = [];
  for (var i = 0, n = out.data32F.length; i < n; i += 7) {
    var confidence = out.data32F[i + 2];
    var left = out.data32F[i + 3] * img.cols;
    var top = out.data32F[i + 4] * img.rows;
    var right = out.data32F[i + 5] * img.cols;
    var bottom = out.data32F[i + 6] * img.rows;
    left = Math.min(Math.max(0, left), img.cols - 1);
    right = Math.min(Math.max(0, right), img.cols - 1);
    bottom = Math.min(Math.max(0, bottom), img.rows - 1);
    top = Math.min(Math.max(0, top), img.rows - 1);

    if (confidence > 0.5 && left < right && top < bottom) {
      faces.push({x: left, y: top, width: right - left, height: bottom - top})
    }
  }
  blob.delete();
  out.delete();
  return faces;
};
//! [Run face detection model]

//! [Get 128 floating points feature vector]
function face2vec(face) {
  var blob = cv.blobFromImage(face, 1.0 / 255, {width: 96, height: 96}, [0, 0, 0, 0], true, false)
  netRecogn.setInput(blob);
  var vec = netRecogn.forward();
  blob.delete();
  return vec;
};
//! [Get 128 floating points feature vector]

//! [Recognize]
function recognize(face) {
  var vec = face2vec(face);

  var bestMatchName = 'unknown';
  var bestMatchScore = 0.5;  // Actually, the minimum is -1 but we use it as a threshold.
  for (name in persons) {
    var personVec = persons[name];
    var score = vec.dot(personVec);
    if (score > bestMatchScore) {
      bestMatchScore = score;
      bestMatchName = name;
    }
  }
  vec.delete();
  return bestMatchName;
};
//! [Recognize]

function loadModels(callback) {
  var utils = new Utils('');
  var proto = 'https://raw.githubusercontent.com/opencv/opencv/master/samples/dnn/face_detector/deploy_lowres.prototxt';
  var weights = 'https://raw.githubusercontent.com/opencv/opencv_3rdparty/dnn_samples_face_detector_20180205_fp16/res10_300x300_ssd_iter_140000_fp16.caffemodel';
  var recognModel = 'https://raw.githubusercontent.com/pyannote/pyannote-data/master/openface.nn4.small2.v1.t7';
  utils.createFileFromUrl('face_detector.prototxt', proto, () => {
    document.getElementById('status').innerHTML = 'Downloading face_detector.caffemodel';
    utils.createFileFromUrl('face_detector.caffemodel', weights, () => {
      document.getElementById('status').innerHTML = 'Downloading OpenFace model';
      utils.createFileFromUrl('face_recognition.t7', recognModel, () => {
        document.getElementById('status').innerHTML = '';
        netDet = cv.readNetFromCaffe('face_detector.prototxt', 'face_detector.caffemodel');
        netRecogn = cv.readNetFromTorch('face_recognition.t7');
        callback();
      });
    });
  });
};

function main() {
  // Create a camera object.
  var output = document.getElementById('output');
  var camera = document.createElement("video");
  camera.setAttribute("width", output.width);
  camera.setAttribute("height", output.height);

  // Get a permission from user to use a camera.
  navigator.mediaDevices.getUserMedia({video: true, audio: false})
    .then(function(stream) {
      camera.srcObject = stream;
      camera.onloadedmetadata = function(e) {
        camera.play();
      };
  });

  //! [Open a camera stream]
  var cap = new cv.VideoCapture(camera);
  var frame = new cv.Mat(camera.height, camera.width, cv.CV_8UC4);
  var frameBGR = new cv.Mat(camera.height, camera.width, cv.CV_8UC3);
  //! [Open a camera stream]

  //! [Add a person]
  document.getElementById('addPersonButton').onclick = function() {
    var rects = detectFaces(frameBGR);
    if (rects.length > 0) {
      var face = frameBGR.roi(rects[0]);

      var name = prompt('Say your name:');
      var cell = document.getElementById("targetNames").insertCell(0);
      cell.innerHTML = name;

      persons[name] = face2vec(face).clone();

      var canvas = document.createElement("canvas");
      canvas.setAttribute("width", 96);
      canvas.setAttribute("height", 96);
      var cell = document.getElementById("targetImgs").insertCell(0);
      cell.appendChild(canvas);

      var faceResized = new cv.Mat(canvas.height, canvas.width, cv.CV_8UC3);
      cv.resize(face, faceResized, {width: canvas.width, height: canvas.height});
      cv.cvtColor(faceResized, faceResized, cv.COLOR_BGR2RGB);
      cv.imshow(canvas, faceResized);
      faceResized.delete();
    }
  };
  //! [Add a person]

  //! [Define frames processing]
  var isRunning = false;
  const FPS = 30;  // Target number of frames processed per second.
  function captureFrame() {
    var begin = Date.now();
    cap.read(frame);  // Read a frame from camera
    cv.cvtColor(frame, frameBGR, cv.COLOR_RGBA2BGR);

    var faces = detectFaces(frameBGR);
    faces.forEach(function(rect) {
      cv.rectangle(frame, {x: rect.x, y: rect.y}, {x: rect.x + rect.width, y: rect.y + rect.height}, [0, 255, 0, 255]);

      var face = frameBGR.roi(rect);
      var name = recognize(face);
      cv.putText(frame, name, {x: rect.x, y: rect.y}, cv.FONT_HERSHEY_SIMPLEX, 1.0, [0, 255, 0, 255]);
    });

    cv.imshow(output, frame);

    // Loop this function.
    if (isRunning) {
      var delay = 1000 / FPS - (Date.now() - begin);
      setTimeout(captureFrame, delay);
    }
  };
  //! [Define frames processing]

  document.getElementById('startStopButton').onclick = function toggle() {
    if (isRunning) {
      isRunning = false;
      document.getElementById('startStopButton').innerHTML = 'Start';
      document.getElementById('addPersonButton').disabled = true;
    } else {
      function run() {
        isRunning = true;
        captureFrame();
        document.getElementById('startStopButton').innerHTML = 'Stop';
        document.getElementById('startStopButton').disabled = false;
        document.getElementById('addPersonButton').disabled = false;
      }
      if (netDet == undefined || netRecogn == undefined) {
        document.getElementById('startStopButton').disabled = true;
        loadModels(run);  // Load models and run a pipeline;
      } else {
        run();
      }
    }
  };

  document.getElementById('startStopButton').disabled = false;
};
</script>

</head>

<body onload="cv['onRuntimeInitialized']=()=>{ main() }">
  <button id="startStopButton" type="button" disabled="true">Start</button>
  <div id="status"></div>
  <canvas id="output" width=640 height=480 style="max-width: 100%"></canvas>

  <table>
    <tr id="targetImgs"></tr>
    <tr id="targetNames"></tr>
  </table>
  <button id="addPersonButton" type="button" disabled="true">Add a person</button>
</body>

</html>
<ol type="1">
<li>Run face detection network to detect faces on input image. <div class="fragment"><div class="line">function detectFaces(img) {</div><div class="line">  var blob = cv.blobFromImage(img, 1, {width: 192, height: 144}, [104, 117, 123, 0], false, false);</div><div class="line">  netDet.setInput(blob);</div><div class="line">  var out = netDet.forward();</div><div class="line"></div><div class="line">  var faces = [];</div><div class="line">  for (var i = 0, n = out.data32F.length; i &lt; n; i += 7) {</div><div class="line">    var confidence = out.data32F[i + 2];</div><div class="line">    var left = out.data32F[i + 3] * img.cols;</div><div class="line">    var top = out.data32F[i + 4] * img.rows;</div><div class="line">    var right = out.data32F[i + 5] * img.cols;</div><div class="line">    var bottom = out.data32F[i + 6] * img.rows;</div><div class="line">    left = Math.min(Math.max(0, left), img.cols - 1);</div><div class="line">    right = Math.min(Math.max(0, right), img.cols - 1);</div><div class="line">    bottom = Math.min(Math.max(0, bottom), img.rows - 1);</div><div class="line">    top = Math.min(Math.max(0, top), img.rows - 1);</div><div class="line"></div><div class="line">    if (confidence &gt; 0.5 &amp;&amp; left &lt; right &amp;&amp; top &lt; bottom) {</div><div class="line">      faces.push({x: left, y: top, width: right - left, height: bottom - top})</div><div class="line">    }</div><div class="line">  }</div><div class="line">  blob.delete();</div><div class="line">  out.delete();</div><div class="line">  return faces;</div><div class="line">};</div></div><!-- fragment -->You may play with input blob sizes to balance detection quality and efficiency. The bigger input blob the smaller faces may be detected.</li>
<li>Run face recognition network to receive <code>128</code>-dimensional unit feature vector by input face image. <div class="fragment"><div class="line">function face2vec(face) {</div><div class="line">  var blob = cv.blobFromImage(face, 1.0 / 255, {width: 96, height: 96}, [0, 0, 0, 0], true, false)</div><div class="line">  netRecogn.setInput(blob);</div><div class="line">  var vec = netRecogn.forward();</div><div class="line">  blob.delete();</div><div class="line">  return vec;</div><div class="line">};</div></div><!-- fragment --></li>
<li>Perform a recognition. <div class="fragment"><div class="line">function recognize(face) {</div><div class="line">  var vec = face2vec(face);</div><div class="line"></div><div class="line">  var bestMatchName = &#39;unknown&#39;;</div><div class="line">  var bestMatchScore = 0.5;  // Actually, the minimum is -1 but we use it as a threshold.</div><div class="line">  for (name in persons) {</div><div class="line">    var personVec = persons[name];</div><div class="line">    var score = vec.dot(personVec);</div><div class="line">    if (score &gt; bestMatchScore) {</div><div class="line">      bestMatchScore = score;</div><div class="line">      bestMatchName = name;</div><div class="line">    }</div><div class="line">  }</div><div class="line">  vec.delete();</div><div class="line">  return bestMatchName;</div><div class="line">};</div></div><!-- fragment -->Match a new feature vector with registered ones. Return a name of the best matched person.</li>
<li>The main loop. <div class="fragment"><div class="line">  var isRunning = false;</div><div class="line">  const FPS = 30;  // Target number of frames processed per second.</div><div class="line">  function captureFrame() {</div><div class="line">    var begin = Date.now();</div><div class="line">    cap.read(frame);  // Read a frame from camera</div><div class="line">    cv.cvtColor(frame, frameBGR, cv.COLOR_RGBA2BGR);</div><div class="line"></div><div class="line">    var faces = detectFaces(frameBGR);</div><div class="line">    faces.forEach(function(rect) {</div><div class="line">      cv.rectangle(frame, {x: rect.x, y: rect.y}, {x: rect.x + rect.width, y: rect.y + rect.height}, [0, 255, 0, 255]);</div><div class="line"></div><div class="line">      var face = frameBGR.roi(rect);</div><div class="line">      var name = recognize(face);</div><div class="line">      cv.putText(frame, name, {x: rect.x, y: rect.y}, cv.FONT_HERSHEY_SIMPLEX, 1.0, [0, 255, 0, 255]);</div><div class="line">    });</div><div class="line"></div><div class="line">    cv.imshow(output, frame);</div><div class="line"></div><div class="line">    // Loop this function.</div><div class="line">    if (isRunning) {</div><div class="line">      var delay = 1000 / FPS - (Date.now() - begin);</div><div class="line">      setTimeout(captureFrame, delay);</div><div class="line">    }</div><div class="line">  };</div></div><!-- fragment -->A main loop of our application receives a frames from a camera and makes a recognition of an every detected face on the frame. We start this function ones when OpenCV.js was initialized and deep learning models were downloaded. </li>
</ol>
</div></div><!-- contents -->
<!-- HTML footer for doxygen 1.8.6-->
<!-- start footer part -->
<hr class="footer"/><address class="footer"><small>
Generated on Fri Apr 2 2021 11:36:34 for OpenCV by &#160;<a href="http://www.doxygen.org/index.html">
<img class="footer" src="../../doxygen.png" alt="doxygen"/>
</a> 1.8.13
</small></address>
<script type="text/javascript">
//<![CDATA[
addTutorialsButtons();
//]]>
</script>
</body>
</html>
